# -*- coding: utf-8 -*-
import math
import os
import random
import re
from datetime import datetime, timedelta
import requests
import json
import shutil
import time
from urllib.parse import quote
import string
import xlrd
import xlsxwriter

""" Get POI data from GaodeMap
    Create by SidChen in 2020.11
    Contact at sidchen0@qq.com"""


# check key
def key_check(p):
    def valid(key):
        url = 'http://restapi.amap.com/v3/place/polygon?polygon=107.75776863,23.10758231;108.3548069,22.75358574&' \
              'types=060000&key={}&extensions=all&offset=5&page=1'.format(key)
        request = requests.get(url=url, timeout=(5, 27))
        html = json.loads(request.text)
        request.close()
        print(html['info'])
        return html['info']

    print('* 第一步：确认key *')
    print('正在读取"{}"文件（高德web服务端API接口）\n* 可粘贴多个账号下的key，一行一个'.format(p))
    file_exist(p)
    k = open(p, 'r', encoding='utf-8-sig')
    keys2 = set()
    for key in k:
        keys2.add(key.strip('\n'))
    i = 0
    for key in keys2:
        i += 1
        info = valid(key)
        if info != 'OK' and i == len(keys2):
            pop = 'key出错，原因：{}，回车退出'.format(info)
            wrong_unexpected(pop)
        elif info == 'OK':
            print('Key {} 读取成功，确认有效!'.format(i))
            keys.append(key)
    return


def file_exist(path):
    """判断关键文件是否存在或为空"""
    if os.path.exists(path):
        fl = open(path, 'r', encoding='utf-8')
        print('读取文件中…………')
        if len(fl.read()) == 0:
            print(path + '文件为空，请检查。')
            pop = '回车退出'
            wrong_unexpected(pop)
        else:
            return 'false'
    else:
        print(path + '文件不存在，请检查。')
        pop = '回车退出'
        wrong_unexpected(pop)


# recreate a file
def folder_delete(p):
    if os.path.exists(p):
        print('检测到"{}"文件夹，为避免重复写入数据将自动删除'.format(p))
        try:
            shutil.rmtree(p)
        except:
            pop = '{}文件夹下可能有打开的文件，请关闭后再运行程序，回车退出'.format(p)
            wrong_unexpected(pop)
    if not os.path.exists(p):
        os.makedirs(p)


# file must existence
def folder_exist(p):
    if not os.path.exists(p):
        pop = '"{}"文件不存在，请确认，回车退出\n'.format(p)
        wrong_unexpected(pop)
    else:
        return True


def input_wrong(file, correct):
    print('\n\n{}内容格式不正确，请确认，回车退出'.format(file))
    pop = correct
    wrong_unexpected(pop)
    return


def input_wait(remind, defaut):
    """input or leap over"""
    # CODE from: https://blog.csdn.net/weixin_39858881/article/details/107152961
    import func_timeout

    @func_timeout.func_set_timeout(15)
    def askChoice():
        return input(remind)

    try:
        s = askChoice()
    except func_timeout.exceptions.FunctionTimedOut as e:
        s = defaut
    print('\n', s)
    return s


def CutChina(ml, lgt1, lat1, lgt2, lat2, types):
    """切割地图保证能够获得该区域所有POI数据"""
    url = base_url + str(lgt1) + "," + str(lat1) + ";" + str(lgt2) + "," + str(
        lat2) + "&types=" + types + "&key=" + keys[random.randint(0, len(keys) - 1)] + "&extensions=all&offset=5&page=1"
    print(url)
    file = ml + types + ".txt"
    data = DownloadHtml(url=url)
    jsonData = json.loads(data)
    count = int(jsonData["count"])
    if count != 0:
        print(count)
        if 0 < count < 301:
            with open(file, "a") as f:
                f.writelines('{},{},{},{}\n'.format(lgt1, lat1, lgt2, lat2))
                human()
            print("结果已写入" + file)
            cut_num[0] += 1
            cut_num[1] += count
        elif count > 300:
            print('* too much data, re-dividing......')
            # the five points and four recs
            midX = (lgt1 + lgt2) / 2
            midY = (lat1 + lat2) / 2
            rec1 = [(lgt1, lat1), (lgt1, midY), (midX, midY), (midX, lat1)]
            rec2 = [(lgt2, lat2), (lgt2, midY), (midX, midY), (midX, lat2)]
            rec3 = [(lgt1, midY), (lgt1, lat2), (midX, midY), (midX, lat2)]
            rec4 = [(lgt2, lat1), (lgt2, midY), (midX, midY), (midX, lat1)]
            if region_mode == 2:
                # neglect the outside region
                def equal(a, b):
                    # not in the original list
                    d = False
                    if a in rg_lgt:
                        c = rg_lgt.index(a)
                        if b == rg_lat[c]:
                            d = True
                    return d

                def radiation(a, b):
                    # check whether the point is inside the original region
                    # 1. divide the points according to the input point
                    rg_divide = {}
                    for i in range(0, len(rg_lgt)):
                        vd = rg_lgt[i] - a
                        # draw a radiation and count the crossing; odd-inside, even-outside
                        if vd > 0:
                            rg_key = str(rg_lgt[i]) + ',' + str(rg_lat[i])
                            # horizontal division, vertical division
                            rg_divide[rg_key] = (i, rg_lat[i] - b, vd)
                    # rg_ra = [('lgt, lat', (id, y-b, x-a))]
                    rg_ra = list(rg_divide.items())
                    rg_ra.sort(key=lambda x: x[1])

                    # 2. find the crossing and update the tick
                    cs_num = 0
                    t = 0
                    while t < (len(rg_ra) - 1):
                        # first point- based on vertical division
                        i = rg_ra[t][1][1]
                        for j in range(t + 1, len(rg_ra)):
                            # looking for another goal point
                            k = rg_ra[j][1][1]
                            t += 1
                            if i * k <= 0:
                                cs_num += 1
                                break
                    # 3. inside or outside?
                    if (cs_num % 2) != 0:
                        rt = 'inside'
                    else:
                        rt = 'outside'
                    return rt

                # check all the points
                i = (lgt1, lgt2, midX)
                j = (lat1, lat2, midY)
                points = []
                for m in i:
                    for n in j:
                        points.append((m, n))
                points_check = {}
                for k in points:
                    if equal(k[0], k[1]):
                        points_check[k] = 'in'
                    else:
                        l = radiation(k[0], k[1])
                        if l == 'inside':
                            points_check[str(k)] = 'in'
                        else:
                            points_check[str(k)] = 'out'

                # continue to divide the original region
                def dropping(rec):
                    ck = 0
                    for i in rec:
                        if points_check[str(i)] == 'out':
                            ck += 1
                    if ck != 4:
                        # inside
                        CutChina(ml, rec[0][0], rec[0][1], rec[2][0], rec[2][1], types)
                    else:
                        print('* region is totally outside and pass:\n* {}'.format(rec))
                dropping(rec1)
                dropping(rec2)
                dropping(rec3)
                dropping(rec4)
            else:
                CutChina(ml, lgt1, lat1, midX, midY, types)
                CutChina(ml, lgt2, lat2, midX, midY, types)
                CutChina(ml, lgt1, lat2, midX, midY, types)
                CutChina(ml, lgt2, lat1, midX, midY, types)
    else:
        print(jsonData['info'])


def DownloadHtml(url):
    human()
    # 伪装user-agent，因为联网不稳定所以避免使用fake_useragent模块
    # 参考：https://blog.csdn.net/weixin_44818729/article/details/107060218
    browsers = {
        "chrome": [
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.222"
            "7.1 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41."
                                 "0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 "
                                                           "(KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/53"
            "7.36", "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 S"
                    "afari/537.36", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chro"
                                    "me/41.0.2225.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
            "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
            "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36",
            "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36",
            "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
            "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14"],
        "opera": ["Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16",
                  "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
                  "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14",
                  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14",
                  "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02",
                  "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
                  "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00",
                  "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
                  "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00",
                  "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0",
                  "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62",
                  "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62",
                  "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
                  "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52",
                  "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51",
                  "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51",
                  "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50",
                  "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50",
                  "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11",
                  "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11",
                  "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11",
                  "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10",
                  "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10",
                  "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10",
                  "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1",
                  "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01",
                  "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01",
                  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01",
                  "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
                  "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01",
                  "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01",
                  "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00",
                  "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00"],
        "firefox": ["Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1",
                    "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0",
                    "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0",
                    "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0",
                    "Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101  Firefox/28.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0",
                    "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0",
                    "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/23.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0",
                    "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:22.0) Gecko/20130328 Firefox/22.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20130405 Firefox/22.0",
                    "Mozilla/5.0 (Microsoft Windows NT 6.2.9200.0); rv:22.0) Gecko/20130405 Firefox/22.0",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1",
                    "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20130514 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.2; rv:21.0) Gecko/20130326 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130401 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130328 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130401 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 5.0; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0",
                    "Mozilla/5.0 (Windows NT 6.2; Win64; x64;) Gecko/20100101 Firefox/20.0",
                    "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/19.0",
                    "Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/18.0.1",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0)  Gecko/20100101 Firefox/18.0",
                    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6"],
        "internetexplorer": ["Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko",
                             "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0;  rv:11.0) like Gecko",
                             "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)",
                             "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)",
                             "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)",
                             "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
                             "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)",
                             "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
                             "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; FunWebProducts)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)",
                             "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)",
                             "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)",
                             "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)",
                             "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 3.0)"],
        "safari": [
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A",
            "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
            "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; sv-se) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ko-kr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; it-it) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-fr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; es-es) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-gb) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; sv-SE) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; hu-HU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; it-IT) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-ch) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; ar) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Android 2.2; Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; tr-TR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5",
            "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-cn) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5"]}
    from random import choice
    ag = choice(browsers[choice(list(browsers.keys()))])
    headers = {
        # 伪装
        'User-Agent': ag
    }
    # 请求
    # 解决请求路径中含义中文或特殊字符
    url = quote(url, safe=string.printable)
    try:
        request = requests.get(url=url, headers=headers, timeout=(5, 27))
        html = request.text
        request.close()
        return html
    except:
        print('{}网页响应错误，30s后自动无限重试……\n（如果疯狂重试不成功，请手动关闭脚本，确认账号后再运行）'.format(datetime.now()))
        time.sleep(30)
        DownloadHtml(url)


# pretend to be human not machine
def human():
    def trans(t1):
        # from str to datetime
        t2 = datetime.strptime(t1, '%H:%M')
        return t2

    def trans2(t3):
        # select %H:%M part from datetime
        t3 = t3.strftime('%H:%M')
        t4 = datetime.strptime(t3, '%H:%M')
        return t4

    tn = datetime.now()
    # beyond 10 minutes take a nap
    pause1 = nap['nap'] + timedelta(minutes=random.randint(10, 15))
    # beyond 1 hour have a sleep
    pause2 = nap['sleep'] + timedelta(minutes=random.randint(60, 90))
    # during lunch or dinner stop here
    pause3 = [('12:00', '12:30'), ('17:30', '18:00')]
    # sleep time
    ts = 0
    mode = ''
    # whether it's time to eat something
    for i in pause3:
        if trans(i[0]) < trans2(tn) < trans(i[1]):
            ts = random.uniform(600, 1200)
            mode = 'dining'
            break
        else:
            mode = 'normal'
    if mode == 'normal':
        if tn < pause1:
            ts = random.uniform(0.5, 3)
        elif pause1 < tn < pause2:
            ts = random.uniform(10, 120)
            nap['nap'] = tn
        elif tn > pause2:
            ts = random.uniform(300, 600)
            nap['sleep'] = tn
    # whether it's late at night
    if trans('0:00') < trans2(tn) < trans('7:00'):
        mode = 'Late at night'
        ts = ts * tn.hour / 2
    # pause
    if ts > 5:
        print('……{}:休眠（{}s）……{}'.format(mode, round(ts, 2), tn))
    time.sleep(ts)


# get each divided map's POI data
def getPOIdata(lgt1, lat1, lgt2, lat2, types):
    if not os.path.exists(pause):
        global total_record
        # get first page data
        json_data = get_data(1, lgt1, lat1, lgt2, lat2, types)
        # total_record: this divided map's POI count
        # range(2, page_number) dosen't contain page_number
        # page_size = 15, not more than 20 as Baidu request
        if (total_record % 20) != 0:
            page_number = int(total_record / 20) + 2
        else:
            page_number = int(total_record / 20) + 1
        # start get data
        with open(json_name, 'w') as f:
            # 去除最后]
            f.write(json.dumps(json_data).rstrip(']'))
            for each_page in range(2, page_number):
                if not os.path.exists(pause):
                    # to get all pages and save in the same json file
                    html0 = get_data(each_page, lgt1, lat1, lgt2, lat2, types)
                    if html0 != '':
                        html = json.dumps(html0).lstrip('[').rstrip(']')
                        if html:
                            html = "," + html
                        f.write(html)
                        print('已保存到json文件：' + json_name)
                    else:
                        pass
                else:
                    # each page contains 15 POI data
                    cut_num[2] = cut_num[2] - 20 * each_page
                    # save temporary data
                    rec2 = str(lgt1) + "," + str(lat1) + "," + str(lgt2) + "," + str(lat2)
                    remains2.append((types, rec2, each_page, page_number))
                    break
            f.write(']')
            # remained data
            cut_num[2] = cut_num[1] - total_record
    else:
        # get the rest pages
        json_data = get_data(int(remains2[2]), lgt1, lat1, lgt2, lat2, types)
        with open(json_name, 'w') as f:
            # 去除最后]
            f.write(json.dumps(json_data).rstrip(']'))
            for each_page in range(int(remains2[2]) + 1, int(remains2[3])):
                html0 = get_data(each_page, lgt1, lat1, lgt2, lat2, types)
                if html0 != '':
                    html = json.dumps(html0).lstrip('[').rstrip(']')
                    if html:
                        html = "," + html
                    f.write(html)
                    print('已保存到json文件：' + json_name)
                else:
                    pass
            f.write(']')
    print('# 已获取{}类{},{}|{},{}区域的POI数据, 共{}个，剩余{}个，预计剩余时间{}min'
          .format(types, lgt1, lat1, lgt2, lat2, total_record, cut_num[2], cut_num[2] / 250))


# 获取数据
def get_data(pageindex, lgt1, lat1, lgt2, lat2, types):
    global total_record
    print('解析页码： ' + str(pageindex) + ' ... ...')
    url = base_url + str(lgt1) + "," + str(lat1) + "," + str(lgt2) + "," + str(lat2) + "&types=" \
          + types + "&key=" + keys[random.randint(0, len(keys) - 1)] + "&extensions=all&offset=20&page=" + str(
        pageindex)
    html = DownloadHtml(url)
    try:
        rr = json.loads(html)
        if total_record == 0:
            total_record = int(rr['count'])
        # sometimes ti's empty
        if len(rr['pois']) == 0:
            print('* 无数据 *')
        return rr['pois']
    except:
        wrong = '* 取回数据不是目标格式,类型：{}，区域:{},{}|{},{}'.format(types, lgt1, lat1, lgt2, lat2)
        wrongs.append(wrong)
        print(wrong)
        return ''


# 写入数据到excel
def write_data_to_excel(name):
    # 获取当前日期
    today = datetime.today()
    # 将获取到的datetime对象仅取日期
    today_date = datetime.date(today)
    # 从文件中读取数据
    fp = open(json_name, 'r')
    result = json.loads(fp.read())
    # 整合POI基本数据和详情页数据
    for i in result:
        poi = []
        for m in bkeys:
            # split and convert coordination
            # 'location' :116.463368,40.000511
            if m == 'location' and len(i[m]) != 0:
                lgt, lat = i[m].split(',')
                lgt84, lat84 = gcj02towgs84(float(lgt), float(lat))
                poi.append(lgt)
                poi.append(lat)
                poi.append(lgt84)
                poi.append(lat84)
            elif m == 'biz_ext':
                try:
                    poi.append(float(i[m]['rating']))
                    poi.append(float(i[m]['cost']))
                except:
                    poi.append('-')
                    poi.append('-')
            else:
                try:
                    poi.append(i[m])
                except:
                    poi.append('-')
        pois.append(poi)
    # 实例化一个Workbook()对象(即excel文件)
    f = name + str(today_date) + '.xlsx'
    wbk = xlsxwriter.Workbook(f)
    # 新建一个名为Sheet1的excel sheet。
    sheet = wbk.add_worksheet('Sheet1')
    # 创建表头 - 不需要的!
    # for循环访问并获取数组下标enumerate函数
    # for index, hkey in enumerate(hkeys):
    #    sheet.write(0, index, hkey)
    # 遍历result中的每个元素
    for i in range(len(pois)):
        for j in range(len(pois[i])):
            try:
                sheet.write(i, j, float(pois[i][j]))
            except:
                sheet.write(i, j, str(pois[i][j]))
    wbk.close()
    print('保存到excel文件： ' + name + str(today_date) + '.xlsx!')


# 打开一个excel文件
def open_xls(file):
    fh = xlrd.open_workbook(file)
    return fh


# 获取excel中所有的sheet表
def getsheet(fh):
    return fh.sheets()


# 获取sheet表的行数
def getnrows(fh, sheet):
    table = fh.sheets()[sheet]
    return table.nrows


# 读取文件内容并返回行内容
def getFilect(file, shnum):
    fh = open_xls(file)
    table = fh.sheets()[shnum]
    num = table.nrows
    for row in range(num):
        rdata = table.row_values(row)
        datavalue.append(rdata)
    return datavalue


# 获取sheet表的个数
def getshnum(fh):
    x = 0
    sh = getsheet(fh)
    for sheet in sh:
        x += 1
    return x


def all_file(path_par):
    """ return all the file name under one folder"""
    path_list = []
    # 三个参数：分别返回1.父目录 2.所有文件夹名字（不含路径） 3.所有文件名字
    for parent, dirnames, filenames in os.walk(path_par):
        for filename in filenames:
            c = os.path.join(parent, filename)
            path_list.append(c)
    return path_list


def gcj02towgs84(lng, lat):
    """
    GCJ02(火星坐标系)转GPS84
    :param lng:火星坐标系的经度
    :param lat:火星坐标系纬度
    :return:
    """

    def transformlat(lng, lat):
        ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + \
              0.1 * lng * lat + 0.2 * math.sqrt(math.fabs(lng))
        ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
                math.sin(2.0 * lng * pi)) * 2.0 / 3.0
        ret += (20.0 * math.sin(lat * pi) + 40.0 *
                math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
        ret += (160.0 * math.sin(lat / 12.0 * pi) + 320 *
                math.sin(lat * pi / 30.0)) * 2.0 / 3.0
        return ret

    def transformlng(lng, lat):
        ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + \
              0.1 * lng * lat + 0.1 * math.sqrt(math.fabs(lng))
        ret += (20.0 * math.sin(6.0 * lng * pi) + 20.0 *
                math.sin(2.0 * lng * pi)) * 2.0 / 3.0
        ret += (20.0 * math.sin(lng * pi) + 40.0 *
                math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
        ret += (150.0 * math.sin(lng / 12.0 * pi) + 300.0 *
                math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
        return ret

    def out_of_china(lng, lat):
        """
        判断是否在国内，不在国内不做偏移
        :param lng:
        :param lat:
        :return:
        """
        if lng < 72.004 or lng > 137.8347:
            return True
        if lat < 0.8293 or lat > 55.8271:
            return True
        return False

    x_pi = 3.14159265358979324 * 3000.0 / 180.0
    pi = 3.1415926535897932384626  # π
    a = 6378245.0  # 长半轴
    ee = 0.00669342162296594323  # 扁率

    if out_of_china(lng, lat):
        return lng, lat
    dlat = transformlat(lng - 105.0, lat - 35.0)
    dlng = transformlng(lng - 105.0, lat - 35.0)
    radlat = lat / 180.0 * pi
    magic = math.sin(radlat)
    magic = 1 - ee * magic * magic
    sqrtmagic = math.sqrt(magic)
    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
    dlng = (dlng * 180.0) / (a / sqrtmagic * math.cos(radlat) * pi)
    mglat = lat + dlat
    mglng = lng + dlng
    return [lng * 2 - mglng, lat * 2 - mglat]


# sent email for debug or ending notes
def email(receivers, subject, content):
    import smtplib
    from email.mime.text import MIMEText
    # 设置服务器所需信息
    # 163邮箱服务器地址
    global message
    mail_host = 'smtp.163.com'
    # 163用户名
    mail_user = 'sidchen0122'
    # 密码(部分邮箱为授权码)
    mail_pass = 'DQLIJFZTEZMSWWAY'
    # 邮件发送方邮箱地址
    sender = 'sidchen0122@163.com'
    # 邮件内容设置
    content = content + '\n\n可直接回复本邮件获取有限技术支持（错误文件需一并发送，回复不快的）- 公众号[规划人在路上]出品'
    message = MIMEText(content, 'plain', 'utf-8')
    # 邮件主题
    message['Subject'] = subject
    # 发送方信息
    message['From'] = sender
    # 接受方信息,多个邮箱用英文“,”分隔
    message['To'] = receivers
    # 登录并发送邮件
    try:
        smtpObj = smtplib.SMTP()
        # 连接到服务器
        smtpObj.connect(mail_host, 25)
        # 登录到服务器
        smtpObj.login(mail_user, mail_pass)
        # 发送
        smtpObj.sendmail(
            sender, receivers, message.as_string())
        # 退出
        smtpObj.quit()
        print('\n邮件发送成功\n')
    except smtplib.SMTPException as e:
        # 打印错误
        print('\n邮件发送失败，原因\n', e)


def wrong_save(p):
    if len(wrongs) != 0:
        f = open(p, 'a', encoding='utf-8-sig')
        for i in wrongs:
            f.write(i + '\n')
        f.close()
        print('取回数据错误：\n', wrongs)


def table_create(table_name, head):
    """create an Excel table and return the sheet"""
    table = xlsxwriter.Workbook(table_name)
    sheet = table.add_worksheet()
    for i, j in enumerate(head):
        sheet.write(0, i, j)
    return table, sheet


# end remark
def end():
    check = input_wait('\n* 检查脚本更新情况，稍候片刻…………\n（无需更新，可按Enter键退出）\n', 'checking......')
    if len(check) > 0:
        update_check(url_update, progress)
    print('\n'
          '\n****************************'
          '\n*数据已全部获取，Enter键退出*'
          '\n****************************\n\n')
    input()
    exit()


def wrong_unexpected(pop):
    """for unexpected conditions"""
    # reminding
    print(pop)
    print('\n请依据以上提示解决问题')
    # update
    check = input_wait('\n* 是否需要检查脚本更新情况，以应对未预料情况？'
                       '\n（无需更新，按Enter键跳过；\n需要更新，输入任意字符后按Enter键）', '')
    if len(check) > 0:
        print('checking......')
        update_check(url_update, progress)
    # further help
    input('\n若问题还未解决，可联系：sidchen0@qq.com\nEnter键退出')
    exit()



def update_check(url, progress):
    import re
    from datetime import datetime
    import webbrowser
    from time import sleep

    # check update
    progress_ud = {}
    # progress_ud = {'name':[version_new, version_old, interval]}
    html = DownloadHtml(url)
    for i in progress:
        print(f'正在检查更新：{i}')
        # progress = {'name': ['version_old', 'download_link']}
        try:
            date = re.search(f'{i}.*"Modified.": "', html, re.S).span()[1]
            date_update = html[date:date + 10].replace('-', '')
            version_new = datetime.strptime(date_update, '%Y%m%d')
            version_old = datetime.strptime(progress[i][0], '%Y%m%d')
            interval = version_new - version_old
            if interval.days > 0:
                progress_ud[f'{i}'] = (date_update, progress[i][0], interval)
                print(f'# 有更新:{date_update} ')
        except:
            print('* 未知错误')
    if len(progress_ud) > 0:
        load = input_wait('需要调起默认浏览器下载最新脚本文件吗？\n(默认需要，Enter退出)\n', 'Yes')
        if len(load) > 0:
            for file in progress_ud:
                # start the default browser to download the new version
                webbrowser.open(progress[file][1], new=2)
    else:
        print('* 无更新内容')


def input_check2(must_in, l, count):
    f = []
    # three chances
    for i in range(4):
        if i == 3:
            input('请重新运行脚本，任意键退出')
            exit()
        r = input()
        # numbers limit
        if len(r) == 0:
            print('输入值为空，剩余输入次数为:', (2 - i))
            continue
        elif len(r) > l:
            print('输入值超出限制，剩余输入次数为:', (2 - i))
            continue
        else:
            rr = r.split(',')
            # avoid same number
            g = 'out'
            rrr = []
            for j in rr:
                # must fit the set mode
                for k in must_in:
                    if int(j) == k:
                        g = 'in'
                        print(j)
                        rrr.append(j)
                        break
                    else:
                        g = 'out'
            if len(rrr) != count:
                g = 'out'
            if g == 'out':
                print('输入错误，剩余输入次数为:', (2 - i))
                continue
            else:
                for m in rrr:
                    print('已检视:', m)
                    f.append(int(m))
                break
    return f


def input_check(must_in, l):
    f = []
    # three chances
    for i in range(4):
        if i == 3:
            input('请重新运行脚本，任意键退出')
            exit()
        r = input()
        try:
            rr = r.split('，')
        except:
            input('* 分隔符号错误，Enter键退出')
            exit()
        # avoid same number
        g = 'out'
        rrr = []
        for j in rr:
            # must fit the set mode
            if j == '':
                rrr.append('')
                g = 'in'
            else:
                if int(j) in must_in:
                    rrr.append(j)
                    g = 'in'
                else:
                    g = 'out'
        if g == 'out':
            print('输入错误，剩余输入次数为:', (2 - i))
            continue
        else:
            for m in rrr:
                print(f'已检视“{m}”')
                try:
                    f.append(int(m))
                except:
                    f.append(m)
            break
    return f


def RegionChoice(names):
    if '市' and '区' in names:
        ct, dt = names.split('市')
        CityInfo = [ct+'市', dt]
    else:
        CityInfo = [names]
    url_city = url_city_base+'&keywords={}&subdistrict=3&extension=base'.format(CityInfo[0])
    print('* finding......')
    city_choice = json.loads(DownloadHtml(url_city))
    region_dic = {}
    county = {}
    # request the three level administration districts
    if len(city_choice['districts']) != 0:
        i = city_choice['districts'][0]
        # one level: province/city/county
        region_dic[i['name']] = [i['adcode']]
        for j in i['districts']:
            # three level
            if len(j['districts']) != 0:
                for k in j['districts']:
                    county[k['name']] = k['adcode']
                region_dic[j['name']] = [j['adcode'], county]
                county = {}
            else:
                # two level
                region_dic[j['name']] = [j['adcode']]

        # chose the goal city regions
        def CityChoice(ci):
            # ci{'name': ['adcode', {county}]}
            # city order and the 'Enter': ''
            k = ['']
            # city name and the 'Enter': ''
            m = ['']
            n = list(ci.keys())
            # print 5 in one line
            for j in range(1, len(n)):
                i = n[j]
                print(f'{j}.{i}; ', end='')
                k.append(j)
                m.append(i)
                j += 1
                if j % 5 == 0:
                    print(end='\n')
            # user's choices
            h = input_check(k, len(str(len(k))))
            return m, h

        def CityRegion(city_name, city_id):
            url_city_region = url_city_base+'&keywords={}&subdistrict=0&extensions=all'.format(city_id)
            print('* finding......')
            crg = json.loads(DownloadHtml(url_city_region))
            try:
                crp = crg['districts'][0]['polyline']
                GoalCity.append((city_name, city_id, crp))
            except:
                print('* 未找到该区域经纬度范围：', city_name)
            return

        # input one level name： province/city/county
        if len(CityInfo) == 1:
            print('* 已找到：{}，adcode：{}'.format(CityInfo[0], region_dic[CityInfo[0]]))
            print('** 需要进一步选择目标区域吗？\n*** 不需要: Enter；'
                  '\n*** 需要: 请输入相应数字+Enter，多个数字用逗号“,”隔开')
            cn, uc = CityChoice(region_dic)
            if (len(uc) == 0) or ((len(uc) == 1) and (uc[0] == '')):
                # one level and done
                print('*** OK!')
                CityRegion(CityInfo[0], region_dic[CityInfo[0]][0])
            else:
                # second level: province-city/county
                print('*** 已选择{}个区域：{}'.format(len(uc), uc))
                print('****需要进一步选择目标区域吗？'
                      '\n**** 不需要: Enter；\n**** 需要: yes+Enter')
                i = input()
                if len(i) == 0:
                    for j in uc:
                        CityRegion(cn[j], region_dic[cn[j]][0])
                # third level: city-county
                else:
                    for i in uc:
                        # for each city
                        cn2 = cn[i]
                        print('* ', cn2)
                        print('***** 请输入相应数字+Enter，多个数字用【中文】逗号“,”隔开')
                        cn3, uc2 = CityChoice(region_dic[cn2][1])
                        # save the county info
                        for j in uc2:
                            cn4 = cn3[j]
                            CityRegion(cn4, region_dic[cn2][1][cn4])
        # input two level name: CityInfo= ['xx市', 'xx区']
        else:
            i = CityInfo[0]
            j = CityInfo[1]
            CityRegion(i+j, region_dic[j])
    else:
        wrong_unexpected('* 输入区域格式有误无法识别，请人工在region.txt文件中键入目标区域经纬度对')
    return


def excel_write(ph, data):
    from math import ceil
    # create a table and write in data
    table = xlsxwriter.Workbook(ph)
    # avoid the rows limit for xlsx:1048576\xls:65536
    table_type = ph.split('.')[-1]
    if table_type == 'xlsx':
        # ceil: 向上取整
        sheet_num = ceil((len(data)-1)/1048000)
    else:
        sheet_num = int(len(data)/6500)
    print(f'* Total Data:{len(data)} rows; Target Sheet:{sheet_num} sheets\n* writing......')
    for s in range(sheet_num):
        sheet = table.add_worksheet(f'sheet{s}')
        if table_type == 'xlsx':
            id_begin = s * 1048000
            id_end = 1048000 + id_begin
        else:
            id_begin = s * 6500
            id_end = 6500 + id_begin
        if id_end > len(data):
            id_end = len(data)-1
        # write head
        sheet.write(0, 0, 'ID')
        for k in range(len(data[0])):
            sheet.write(0, k + 1, data[0][k])
        # write data
        for i in range(id_begin, id_end):
            # write ID column
            sheet.write(i-id_begin+1, 0, i + 1)
            for j in range(len(data[i + 1])):
                # 这是最宽泛的将数字、字符串分开的方法，不过有可能出错，可以考虑结合“判断是否为数字”的代码使用
                try:
                    sheet.write(i-id_begin+1, j + 1, data[i + 1][j])
                except:
                    sheet.write(i-id_begin+1, j + 1, str(data[i + 1][j]))
        print(f'* Have finished:sheet{s}')
    table.close()
    print('###save as ' + ph)


if __name__ == "__main__":
    print('* 本脚本用于获取多个矩形区域内的POI数据信息（高德数据源）')
    print('* 支持断点续传：需要暂停时，在脚本所在文件夹下新建{}文件即可（下次双击运行脚本，自动恢复进程）\n\n'.format('pause'))
    # check and download the updated progress
    progress = {'Get_GaodeMap_POI_Polygon.exe': ['20230218',
                                                 'https://chdeducn-my.sharepoint.com/personal/201541020106_chd_edu_cn/_layouts/15/download.aspx?UniqueId=2d94bd50%2Da6db%2D41d5%2Dbb8e%2D0e94b3ebc1cb'
                                                 '']
        , 'UpdateLogs.txt': ['20230218',
                             'https://chdeducn-my.sharepoint.com/personal/201541020106_chd_edu_cn/_layouts/15/download.aspx?UniqueId=ebe4c419%2D8cfc%2D4456%2Dba13%2Da9851c37fe30'
                             '']
                }
    # url must be the direct sharing link if it's from OneDrive where you can check latest update date
    url_update = 'https://chdeducn-my.sharepoint.com/:f:/g/personal/201541020106_chd_edu_cn/EsYn3Xel_NxJpJ1JYQjkULQBgKWbEFC88ViuDytwAx1b2g?e=kuOPkj'
    for i in progress:
        print(f'* {i}，版本号{progress[i][0]}，')
    print('*** 有限技术支持：sidchen0 @ qq.com ***\n\n')

    # some common data
    # save every cutting maps' POI count
    global total_record
    json_name = 'poi_get/data_Gaodemap.json'
    base_url = "http://restapi.amap.com/v3/place/polygon?polygon="
    # wrong situation
    wrongs = []
    # 获取数据列
    # save temporary POI data
    pois = []
    # save goal city name for data-clear
    goal_city = set()
    # biz_ext：详情页餐饮等类的评分，平均消费
    bkeys = ['id', 'biz_type', 'name', 'type', 'address', 'location', 'pname', 'cityname', 'adname', 'business_area'
        , 'biz_ext']
    detail_info = ['rating', 'cost']
    # cut into how many rectangles? - final request num :
    # (request times/cut map, total num, remain num//to continue, data in goal city, data which goal area isn't defined)
    cut_num = [0, 0, 0, 0, 0]
    # read POI types
    types = []
    # time sleep mechanism to pretend te be human including two level pause
    start = datetime.now()
    nap = {'start': start, 'nap': start, 'sleep': start}
    # pause command: if there is it we should continue last progress
    pause = 'pause.txt'
    restart = 'poi_get/Continue'
    # save temporary data after pausing
    # temporary: region2=[(type, region)], remains2=[type, region, page_begin, page_end]
    region2 = []
    remains2 = []
    # 地图分幅编号
    sub = []

    # 1. check key
    keys = []
    key_check('reference/key.txt')

    # 2-1. check last data file
    if not os.path.exists(pause):
        folder_delete('poi_get')
    else:
        print('\n##########已检测到前次程序暂停痕迹，将继续获取数据\n读取临时文件中……')
        folder_exist('poi_get/continue')
        f1 = restart + '/GoalRegions.txt'
        f2 = restart + '/PagePause.txt'
        # regain the rest from sub-type-region
        if os.path.exists(f1):
            try:
                f12 = open(f1, 'r', encoding='utf-8-sig')
                # to save types
                sub2 = set()
                types2 = set()
                # split info: sub-type-region
                for i in f12:
                    s, j, h = i.strip('\n').split(';')
                    sub2.add(s)
                    types2.add(j)
                    # re-save sub region folders
                    sub_folder = restart + '/' + s
                    if not os.path.exists(sub_folder):
                        os.makedirs(sub_folder)
                    # re-save divided regions
                    fr = open(sub_folder + '/' + j + '.txt', 'a', encoding='utf-8-sig')
                    # re-save types2
                    fr.write(h + '\n')
                    fr.close()
                for i in sub2:
                    sub.append(i)
                for i in types2:
                    types.append(i)
            except:
                f3 = open(pause, 'w')
                f3.close()
                input('"{}"文件处理时出错，请确认，回车退出\n'.format(f1))
                exit()
        # regain the rest from type-region-page_begin-page_end
        if os.path.exists(f2):
            try:
                # get type-region-page_begin-page_end this four data
                total_record = 0
                f22 = open(f2, 'r', encoding='utf-8-sig')
                remains2 = f22.read().strip('\n').split(';')
                rg = remains2[1].split(',')
                print('\n首先获取前次中途退出的页面…………')
                getPOIdata(float(rg[0]), float(rg[1]), float(rg[2]), float(rg[3]), remains2[0])
                if total_record != 0:
                    try:
                        write_data_to_excel("poi_get/data/{}-{}-高德地图"
                                            .format(remains2[0], len(os.listdir('poi_get/data')) + 1))
                    except:
                        print('* 写入表格错误')
                        wrongs.append('{};{}'.format(remains2[0], rg))
                os.remove(pause)
            except:
                f3 = open(pause, 'w')
                f3.close()
                input('"{}"文件处理时出错，请确认，回车退出\n'.format(f2))
                exit()
        else:
            os.remove(pause)
        remains2.clear()
        region2.clear()

    # 2-2 a new start: read regions
    region_format = 'region.txt文件至少需要包含一组完整数据（3行，Enter键换行）' \
                    '\n第一行：编号；' \
                    '\n* 编号可以是数字、中文，但不要是特殊字符，若为真实完整中文城市、区县名称，后期将对数据自动进行筛选' \
                    '\n第二行：左上角点经度，左上角点纬度;' \
                    '\n第三行：右下角点经度，右下角点纬度\n（一对角点确定一个矩形区域）' \
                    '\n如（以下为两组数据）：' \
                    '\n1' \
                    '\n119.353198,26.003456' \
                    '\n119.359635,25.993736' \
                    '\n鼓楼区' \
                    '\n119.355601,25.995665' \
                    '\n119.365385,26.007429\n'
    if not os.path.exists(restart):
        print('\n\n* 第二步： 确定数据获取区域'
              '\n(输入相应数字+Enter键）'
              '\n1:读取本地reference/region.txt文件'
              '\n2:程序自动获取边界：用于爬取某一行政区域的全部数据（区县及以上）')
        region_mode = input_check2([1, 2], 1, 1)[0]
        if region_mode == 2:
            # AutoSetRegion
            # 1. get goal region polyline info
            url_city_base = 'https://restapi.amap.com/v3/config/district?key={}' \
                .format(keys[random.randint(0, len(keys) - 1)])
            # GoalCity[('name', 'adcode', [polygon])]
            GoalCity = []
            i = input('* 请输入目标区域名称(至少输入省级名称）：\n1.xx省/自治区（省级可使用简称，如新疆，输错全称反而无法识别）；'
                      '\n2.xx县/市\n3.xx市xx区；\n4.其他非统一/存在混淆的名称（建议只输入所在省份，后依提示做进一步选择）'
                      '\n* 多个目标区域请用【中文】分号";"隔开\n')
            j = i.split('；')
            for k in j:
                RegionChoice(k)
            # 2. save rec info
            auto_regin = []
            # two pairs of points
            rg_wgs84 = [('CityName', 'lgt_gcj02', 'lat_gcj02', 'lgt_wgs84', 'lat_wgs84')]
            for i in GoalCity:
                rg = i[2]
                a = re.split('[;|]', rg)
                rg_lgt = []
                rg_lat = []
                for j in a:
                    b, c = j.split(',')
                    b = float(b)
                    c = float(c)
                    rg_lgt.append(b)
                    rg_lat.append(c)
                    d = gcj02towgs84(b, c)
                    rg_wgs84.append((i[0], b, c, d[0], d[1]))
                auto_regin.append([i[0], (max(rg_lgt), max(rg_lat)), (min(rg_lgt), min(rg_lat))])
            # save to POIRegion.excel
            excel_write('poi_get/POIRegion.xlsx', rg_wgs84)
            # save to region.txt
            # clear original data if there is
            if os.path.exists('reference/region.txt'):
                os.remove('reference/region.txt')
            # write in new data
            f = open('reference/region.txt', 'a', encoding='utf-8-sig')
            for k in auto_regin:
                f.write(k[0])
                f.write('\n')
                f.write(str(k[1][0])+','+str(k[1][1]))
                f.write('\n')
                f.write(str(k[2][0])+','+str(k[2][1]))
                f.write('\n')
            f.close()
        else:
            print('\n* 确认reference文件夹下有region.txt文件')
            print(region_format)
            print('* 高德坐标拾取网址：https://lbs.amap.com/console/show/picker')
        file_exist('reference/region.txt')
        print('读取region…………')
        s = open('reference/region.txt', 'r', encoding='utf-8-sig')
        # 第一个角点经纬度
        lgt1 = []
        lat1 = []
        # 第二个角点经纬度
        lgt2 = []
        lat2 = []
        r = s.readlines()
        # check region format
        if len(r) % 3 != 0:
            input_wrong('region.txt', '\n错误原因：行数不够\n正确格式:\n' + region_format)
        try:
            for i in range(len(r)):
                j = r[i].strip('\n')
                if i % 3 == 1:
                    x, y = j.split(',')
                    lgt1.append(float(x))
                    lat1.append(float(y))
                elif i % 3 == 2:
                    x, y = j.split(',')
                    lgt2.append(float(x))
                    lat2.append(float(y))
                elif j != '':
                    sub.append(j)
                    goal_city.add(j)
        except:
            input_wrong('region.txt', '\n错误原因：经纬度点“缺经或者少纬”\n正确格式:\n' + region_format)
        s.close()

        print('\n\n* 第三步：读取要获取的POI数据类别 *\n# 确认reference文件夹下有POIClass.txt文件(每类一行)…………')
        POIClass = 'reference/POIClass.txt'
        file_exist(POIClass)
        t = open(POIClass, 'r', encoding='utf-8-sig')
        print('读取POIClass…………')
        for i in t.readlines():
            poi = i.strip('\n')
            if len(poi) > 6:
                input_wrong(POIClass, '格式错误：', poi)
            types.append(poi)
        t.close()

        # cut maps

        print("\n\n* 第四步：分割目标区域，确保获取完整数据 *\n##########开始分割目标区域\n(请耐心等待)")
        # start cut map
        for i in range(len(sub)):
            ml = str('poi_get/MapCutPoint/' + sub[i] + '/')
            os.makedirs(ml)
            for j in range(len(types)):
                CutChina(ml, float(lgt1[i]), float(lat1[i]), float(lgt2[i]), float(lat2[i]), types[j])
                if os.path.exists(ml + types[j] + '.txt'):
                    # 输出该区域存在的POI数据类型列表
                    fme = open(ml + 'POI_Exist.txt', 'a', encoding='utf-8-sig')
                    fme.write(types[j])
                    fme.write('\n')
                    fme.close()
                    # 输出所有区域存在的POI数据类型列表
                    fe = open('poi_get/POI_Exist.txt', 'a', encoding='utf-8-sig')
                    fe.write(str(sub[i] + ':' + types[j]))
                    fe.write('\n')
                    fe.close()
                else:
                    # 输出所有区域不存在的POI数据类型列表
                    print(sub[i] + '区域不存在' + types[j] + '类别的POI数据')
                    fne = open('poi_get/POI_NOTExist.txt', 'a', encoding='utf-8-sig')
                    fne.write(str(sub[i] + ':' + types[j]))
                    fne.write('\n')
                    fne.close()
        # cut map result
        # Based on test:1 minute = 250 POI
        cut_num[2] = cut_num[1]
        print('\n目标数据区域已划分为 {} 个子区域，目标POI数据总量 可能 为 {} 个，至少需要 {} 分钟'
              '\n(时间越长休眠越多，饭点下班，深夜更慢)'
              .format(cut_num[0], cut_num[1], cut_num[2] / 250))
    else:
        try:
            f3 = open(restart + '/AllConditions.txt', 'r', encoding='utf-8-sig')
            f32 = f3.read().strip('\n[]').split(',')
            for i in range(0, 3):
                cut_num[i] = int(f32[i])
            print('\n目标数据区域已划分为 {} 个子区域，目标POI数据总量 可能 为 {} 个，已获取 {} 个，接下来至少需要 {} 分钟'
                  '\n(时间越长休眠越多，饭点下班，深夜更慢)'
                  .format(cut_num[0], cut_num[1], cut_num[2], cut_num[2] / 250))
        except:
            pass
    # ask end reminder
    end_time = datetime.now() + timedelta(minutes=cut_num[2] / 250)
    tomorrow = (end_time.date() - datetime.now().date()).days
    if int(tomorrow) > 0:
        bed_time = input_wait('预计数据获取结束时间为{}，不建议深夜继续爬取数据，需要零点自动停止吗（明天双击程序可以从中断处恢复）？' \
                              '\n1.零点暂停\n2.深夜继续（开发者账号封禁风险）\n'.format(end_time), '1')
    if (cut_num[2] / 250) > 10:
        print('\n数据获取时间较长，是否需要在程序结束时发送邮件提醒(预计结束时间：{})？'.format(end_time))
        receivers = input_wait('需要，请直接输入有效邮箱地址（多个用英文逗号隔开），不需要请回车\n', '')
    else:
        receivers = ''

    # request and save data

    if not os.path.exists('poi_get/data'):
        os.makedirs('poi_get/data')
    # start requesting
    print('\n\n* 第五步：获取POI数据 *\n##########依据分割结果获取POI数据')
    # request by type
    for k in sub:
        if os.path.exists(restart):
            type_region = '{}/{}/'.format(restart, k)
        else:
            type_region = 'poi_get/MapCutPoint/{}/'.format(k)
        for i in types:
            if os.path.exists(type_region + i + '.txt'):
                j = len(os.listdir('poi_get/data'))
                # different zones
                file = open(type_region + i + ".txt", 'r', encoding='utf-8-sig')
                Reclist = file.readlines()
                for recpolygon in Reclist:
                    if not os.path.exists(pause):
                        # clear temporary POI data
                        pois.clear()
                        j += 1
                        pol = recpolygon.strip('\n').split(",")
                        # save every cutting maps' poi count
                        total_record = 0
                        getPOIdata(float(pol[0]), float(pol[1]), float(pol[2]), float(pol[3]), i)
                        if total_record != 0:
                            try:
                                write_data_to_excel(str("poi_get/data/" + i + "-" + str(j) + "-高德地图"))
                            except:
                                print('* 写入表格错误')
                                wrongs.append('{};{};{}'.format(k, i, pol))
                    else:
                        # pause and save temporary data
                        region2.append((k, i, recpolygon))
            else:
                print('未找到{}文件'.format(type_region + i + '.txt'))
    if os.path.exists(pause):
        print('\n\n已检测到暂停程序命令\n正在保存当前进度…………')
        # temporary data must be saved
        if os.path.exists(restart):
            try:
                shutil.rmtree(restart)
            except:
                print('前次临时文件夹无法删除，已创建新文件夹，请手动删除{}，并重命名{}2文件夹为{}'
                      .format(restart, restart, restart))
                restart = restart + '2'
        os.makedirs(restart)
        # save sub-type-region
        if len(region2) != 0:
            f2 = open(restart + '/GoalRegions.txt', 'w', encoding='utf-8-sig')
            for m in region2:
                f2.write(m[0] + ';' + m[1] + ';' + m[2])
            f2.close()
        # save type-region-page_begin-total_record
        if len(remains2) != 0:
            f3 = open(restart + '/PagePause.txt', 'w', encoding='utf-8-sig')
            for m in remains2:
                f3.write(str(m[0]) + ';' + str(m[1]) + ';' + str(m[2]) + ';' + str(m[3]))
            f3.close()
        # save all conditions
        f4 = open(restart + '/AllConditions.txt', 'w', encoding='utf8')
        f4.write(str(cut_num))
        f4.close()
        # save goal city
        f5 = restart + '/GoalCity.txt'
        if not os.path.exists(f5):
            f6 = open(f5, 'w', encoding='utf-8-sig')
            for i in goal_city:
                f6.write(i + ';')
            f6.close()
        print('已获取{}个POI数据，还剩{}个，预计下次需要{}时间'.format(cut_num[1], cut_num[2], cut_num[2] / 250))
        wrong_save('poi_get/POI_NOTExist.txt')
        input('\n保存成功！ 请勿删除poi_get文件夹，下次双击本程序继续中断任务，回车退出\n')
        exit()

    # 开始合并POI数据

    print('\n\n* 第六步：处理数据并保存 *\n##########开始合并POI数据')
    # 定义要合并的excel文件列表
    filepath = r'poi_get/data'
    # read all the files
    allxls = all_file(filepath)
    if len(allxls) == 0:
        pop = '此次未获取到任何POI数据，回车退出\n'
        wrong_unexpected(pop)
    # 存储所有读取的结果
    datavalue = []
    for fl in allxls:
        fh = open_xls(fl)
        # get sheet
        x = getshnum(fh)
        for shnum in range(x):
            print("正在读取文件：" + str(fl) + "的第" + str(shnum + 1) + "个sheet表的内容...")
            # get all the lines
            rvalue = getFilect(fl, shnum)
    # 定义最终合并后生成的新文件
    end_file = 'poi_get/高德POI-未筛选数据-{}.xlsx'.format(datetime.now().date())
    end_file2 = 'poi_get/高德POI-目标城市区域数据-{}.xlsx'.format(datetime.now().date())
    # 写入表头
    hkeys = ['id', '行业类型', '名称', '类型', '地址', '经度GCJ02', '纬度GCJ02', '经度WGS84', '纬度WGS84', '省份名称',
             '城市名称', '区域名称', '所在商圈', '评分', '平均消费']
    wb, ws = table_create(end_file, hkeys)
    wb2, ws2 = table_create(end_file2, hkeys)
    # all the lines
    print('* 开始写入数据…………')
    # 仅保留目标城市数据
    f5 = restart + '/GoalCity.txt'
    if len(goal_city) == 0:
        if os.path.exists(f5):
            f6 = open(f5, 'r', encoding='utf-8sig')
            goal_city = f6.read().split(';')
            f6.close()
        else:
            print('未读取到有效目标城市信息，请在Excel中手动进行筛选')
    # 储存ID字段用于数据去重
    data_check = set()
    # start write in data
    for a in range(len(rvalue)):
        if rvalue[a][0] in data_check:
            pass
        elif rvalue[a][0] == '':
            pass
        # data in goal city
        elif rvalue[a][10] in goal_city or rvalue[a][11] in goal_city:
            data_check.add(rvalue[a][0])
            cut_num[3] += 1
            for b in range(len(rvalue[a])):
                # cell value
                c = rvalue[a][b]
                # write into table
                try:
                    ws2.write(cut_num[3], b, float(c))
                except:
                    ws2.write(cut_num[3], b, str(c))
        # other data
        else:
            data_check.add(rvalue[a][0])
            cut_num[4] += 1
            for b in range(len(rvalue[a])):
                # cell value
                c = rvalue[a][b]
                # write into table
                try:
                    ws.write(cut_num[4], b, float(c))
                except:
                    ws.write(cut_num[4], b, str(c))
    data_total = 0
    if cut_num[3] != 0:
        data_total += cut_num[3]
        wb2.close()
        print('* 已自动筛选目标城市区域数据')
        print("** 文件合并完成，保存为：{}".format(end_file2))
    if cut_num[4] != 0:
        data_total += cut_num[4]
        wb.close()
        print('* 未自动筛选目标数据区域')
        print("** 文件合并完成，保存为：{}".format(end_file))
    print('\n\n##########POI数据爬取完成')
    duration = datetime.now() - nap['start']
    if os.path.exists('poi_get/POI_NOTExist.txt'):
        fne = open('poi_get/POI_NOTExist.txt', 'r', encoding='utf-8-sig')
        result = '已获取全部POI数据，共 {} 个，耗时{} \n目标区域不存在的POI数据类型为：\n{}' \
            .format(data_total, duration, fne.read())
        fne.close()
    else:
        result = '已获取全部POI数据，共 {} 个，耗时{} '.format(data_total, duration)
    wrong_save('poi_get/POI_NOTExist.txt')
    if len(receivers) > 0:
        email(receivers, 'POI数据获取情况', result)
    print(result)
    end()
